/*
 * Copyright 2012 Subhabrata Ghosh <subho.ghosh at langene.net>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */



package net.langene.nlp.pg;

//~--- non-JDK imports --------------------------------------------------------

import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.Segment;
import net.htmlparser.jericho.Source;

import net.langene.nlp.Context;

import org.apache.commons.configuration.ConfigurationException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

//~--- JDK imports ------------------------------------------------------------

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;

import java.net.URL;

import java.sql.Timestamp;

import java.util.Date;
import java.util.LinkedList;
import java.util.List;

//~--- classes ----------------------------------------------------------------

/**
 *
 * @author Subhabrata Ghosh <subho.ghosh at langene.net>
 */
public class PGDownloader {
    private static final long	_SLEEP_INTERVAL_ = 2 * 1000;
    private static final Logger	_logger          = LoggerFactory.getLogger(PGDownloader.class);

    //~--- fields -------------------------------------------------------------

    private List<PGItem>	items  = new LinkedList<PGItem>();
    private String		outdir = null;

    //~--- methods ------------------------------------------------------------

    /**
     * Setup the environment.
     * Performs all initialization tasks.
     * 
     * @throws Exception
     */
    public void setup() throws Exception {
        outdir = Context.get().getConfig().getString("net.langene.pg.outdir");

        if ((outdir == null) || outdir.isEmpty()) {
            throw new ConfigurationException(
                "Invalid Configuration : Missing or empty parameter [net.langene.pg.outdir]");
        }

        _logger.info("Downloaded files will be put in [" + outdir + "]");

        // Load the unprocessed items.
        items = PGDBHelper.getItemRecords(false);
        _logger.info("Loaded [" + items.size() + "] items to be processed...");
    }

    /**
     * Reset the downloaded flag for the records in the DB
     * and delete all downloaded files.
     * 
     * @throws Exception
     */
    public void reset() throws Exception {
        items = PGDBHelper.getItemRecords(true);

        if ((items != null) && (items.size() > 0)) {
            for (PGItem item : items) {
                item.Processed  = false;
                item.ParsedFile = null;
                PGDBHelper.save(item);
            }
        }

        String	houtdir = outdir + "/html/";
        File	hdi     = new File(houtdir);
        File[]	hfiles  = hdi.listFiles();

        if ((hfiles != null) && (hfiles.length > 0)) {
            for (File hf : hfiles) {
                hf.delete();
            }
        }

        houtdir = outdir + "/text/";
        hdi     = new File(houtdir);
        hfiles  = hdi.listFiles();

        if ((hfiles != null) && (hfiles.length > 0)) {
            for (File hf : hfiles) {
                hf.delete();
            }
        }
    }

    /**
     * Download the HTML files and convert them to text.
     * 
     * @throws Exception
     */
    @SuppressWarnings("SleepWhileInLoop")
    public void download() throws Exception {
        if (items.size() <= 0) {
            return;
        }

        for (PGItem item : items) {
            String	outfile = outdir + "/html/" + item.Id + ".html";

            _logger.debug("Downloading file to [" + outfile + "]...");

            long		starttm = new Date().getTime();
            URL			url     = new URL(item.Url);
            BufferedReader	reader  = new BufferedReader(new InputStreamReader(url.openStream()));
            FileOutputStream	fos     = new FileOutputStream(outfile);

            while (true) {
                String	line = reader.readLine();

                if (line == null) {
                    break;
                }

                fos.write(line.getBytes());
            }

            reader.close();
            fos.flush();
            fos.close();
            toText(item);
            item.Processed = true;
            item.Timestamp = new Timestamp(starttm);
            PGDBHelper.save(item);

            long	delta = new Date().getTime() - starttm;

            // Project Gutenberg recommends 2 sec sleep between robot requests.
            if (delta < _SLEEP_INTERVAL_) {
                Thread.sleep(_SLEEP_INTERVAL_ - delta);
            }
        }
    }

    private void toText(PGItem item) throws Exception {
        String	infile  = outdir + "/html/" + item.Id + ".html";
        String	outfile = outdir + "/text/" + item.Id + ".txt";
        Source	src     = new Source(new FileInputStream(infile));

        src.fullSequentialParse();

        FileOutputStream	fos      = new FileOutputStream(outfile);
        List<Element>		elements = src.getAllElements();
        StringBuilder		buffer   = new StringBuilder();

        for (Element elm : elements) {
            if (elm.getStartTag().getName().equals(HTMLElementName.P)) {
                Segment	sg = elm.getContent();

                buffer.append(sg.getRenderer().toString()).append("\n");
            }
        }

        fos.write(buffer.toString().getBytes());
        fos.flush();
        fos.close();
    }
}


//~ Formatted by Jindent --- http://www.jindent.com
